28012b
@@ -20,7 +20,7 @@
 package org.apache.hadoop.hbase.regionserver.handler;
 
 import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -32,7 +32,6 @@
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.util.Progressable;
 import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.Code;
 
 /**
  * Handles opening of a region on a region server.
@@ -46,6 +45,11 @@
public class OpenRegionHandler extends EventHandler {
 
   private final HRegionInfo regionInfo;
 
+  // We get version of our znode at start of open process and monitor it across
+  // the total open. We'll fail the open if someone hijacks our znode; we can
+  // tell this has happened if version is not as expected.
+  private volatile int version = -1;
+
   public OpenRegionHandler(final Server server,
       final RegionServerServices rsServices, HRegionInfo regionInfo) {
     this(server, rsServices, regionInfo, EventType.M_RS_OPEN_REGION);
@@ -73,126 +77,192 @@
public class OpenRegionHandler extends EventHandler {
     }
     final String encodedName = regionInfo.getEncodedName();
 
-    // TODO: Previously we would check for root region availability (but only that it
-    // was initially available, does not check if it later went away)
-    // Do we need to wait on both root and meta to be available to open a region
-    // now since we edit meta?
-
     // Check that this region is not already online
     HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
     if (region != null) {
-      LOG.warn("Attempting open of " + name +
-        " but it's already online on this server");
+      LOG.warn("Attempted open of " + name +
+        " but already online on this server");
       return;
     }
 
-    int openingVersion = transitionZookeeperOfflineToOpening(encodedName);
-    if (openingVersion == -1) return;
+    // If fails, just return.  Someone stole the region from under us.
+    // Calling transitionZookeeperOfflineToOpening initalizes this.version.
+    if (!transitionZookeeperOfflineToOpening(encodedName)) return;
 
-    // Open the region
-    final AtomicInteger openingInteger = new AtomicInteger(openingVersion);
-    try {
-      // Instantiate the region.  This also periodically updates OPENING.
-      region = HRegion.openHRegion(regionInfo, this.rsServices.getWAL(),
-        server.getConfiguration(), this.rsServices,
-        new Progressable() {
-          public void progress() {
-            try {
-              int vsn = ZKAssign.retransitionNodeOpening(
-                  server.getZooKeeper(), regionInfo, server.getServerName(),
-                  openingInteger.get());
-              if (vsn == -1) {
-                throw KeeperException.create(Code.BADVERSION);
-              }
-              openingInteger.set(vsn);
-            } catch (KeeperException e) {
-              server.abort("ZK exception refreshing OPENING node; " + name, e);
-            }
-          }
-        });
-    } catch (IOException e) {
-      LOG.error("Failed open of " + regionInfo +
-        "; resetting state of transition node from OPENING to OFFLINE", e);
-      try {
-        // TODO: We should rely on the master timing out OPENING instead of this
-        // TODO: What if this was a split open?  The RS made the OFFLINE
-        // znode, not the master.
-        ZKAssign.forceNodeOffline(server.getZooKeeper(), regionInfo,
-            server.getServerName());
-      } catch (KeeperException e1) {
-        LOG.error("Error forcing node back to OFFLINE from OPENING; " + name);
-      }
-      return;
+    // Open region.  After a successful open, failures in subsequent processing
+    // needs to do a close as part of cleanup.
+    region = openRegion();
+    if (region == null) return;
+    boolean failed = true;
+    if (tickleOpening("post_region_open")) {
+      if (updateMeta(region)) failed = false;
     }
-    // Region is now open. Close it if error.
-
-    // Re-transition node to OPENING again to verify no one has stomped on us
-    openingVersion = openingInteger.get();
-    try {
-      if ((openingVersion = ZKAssign.retransitionNodeOpening(
-          server.getZooKeeper(), regionInfo, server.getServerName(),
-          openingVersion)) == -1) {
-        LOG.warn("Completed the OPEN of region " + name +
-          " but when transitioning from " +
-          " OPENING to OPENING got a version mismatch, someone else clashed " +
-          "-- closing region");
-        cleanupFailedOpen(region);
-        return;
-      }
-    } catch (KeeperException e) {
-      LOG.error("Failed transitioning node " + name +
-        " from OPENING to OPENED -- closing region", e);
-      cleanupFailedOpen(region);
-      return;
-    } catch (IOException e) {
-      LOG.error("Failed to close region " + name +
-        " after failing to transition -- closing region", e);
+    if (failed) {
       cleanupFailedOpen(region);
       return;
     }
 
-    // Update ZK, ROOT or META
-    try {
-      this.rsServices.postOpenDeployTasks(region,
-        this.server.getCatalogTracker(), false);
-    } catch (IOException e) {
-      LOG.error("Error updating " + name + " location in catalog table -- " +
-        "closing region", e);
-      cleanupFailedOpen(region);
-      return;
-    } catch (KeeperException e) {
-      // TODO: rollback the open?
-      LOG.error("ZK Error updating " + name + " location in catalog " +
-        "table -- closing region", e);
+    if (!transitionToOpened(region)) {
       cleanupFailedOpen(region);
       return;
     }
 
+    // Done!  Successful region open
+    LOG.debug("Opened " + name);
+  }
+
+  /**
+   * Update ZK, ROOT or META.  This can take a while if for example the
+   * .META. is not available -- if server hosting .META. crashed and we are
+   * waiting on it to come back -- so run in a thread and keep updating znode
+   * state meantime so master doesn't timeout our region-in-transition.
+   * Caller must cleanup region if this fails.
+   */
+  private boolean updateMeta(final HRegion r) {
+    // Object we do wait/notify on.  Make it boolean.  If set, we're done.
+    // Else, wait.
+    final AtomicBoolean signaller = new AtomicBoolean(false);
+    PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
+      this.server, this.rsServices, signaller);
+    t.start();
+    int assignmentTimeout = this.server.getConfiguration().
+      getInt("hbase.master.assignment.timeoutmonitor.period", 10000);
+    // Total timeout for meta edit.  If we fail adding the edit then close out
+    // the region and let it be assigned elsewhere.
+    long timeout = assignmentTimeout * 10;
+    long now = System.currentTimeMillis();
+    long endTime = now + timeout;
+    // Let our period at which we update OPENING state to be be 1/3rd of the
+    // regions-in-transition timeout period.
+    long period = Math.max(1, assignmentTimeout/ 3);
+    long lastUpdate = now;
+    while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
+        !this.rsServices.isStopping() && (endTime > now)) {
+      long elapsed = now - lastUpdate;
+      if (elapsed > period) {
+        // Only tickle OPENING if postOpenDeployTasks is taking some time.
+        lastUpdate = now;
+        tickleOpening("post_open_deploy");
+      }
+      synchronized (signaller) {
+        try {
+          signaller.wait(period);
+        } catch (InterruptedException e) {
+          // Go to the loop check.
+        }
+      }
+      now = System.currentTimeMillis();
+    }
+    // Is thread still alive?  We may have left above loop because server is
+    // stopping or we timed out the edit.  Is so, interrupt it.
+    if (t.isAlive()) {
+      LOG.debug("Interrupting thread " + t);
+      t.interrupt();
+    }
+    // Was there an exception opening the region?  This should trigger on
+    // InterruptedException too.  If so, we failed.
+    return t.getException() == null;
+  }
+
+  /**
+   * Thread to run region post open tasks.  Call {@link #getException()} after
+   * the thread finishes to check for exceptions running
+   * {@link RegionServerServices#postOpenDeployTasks(HRegion, org.apache.hadoop.hbase.catalog.CatalogTracker, boolean)}.
+   */
+  static class PostOpenDeployTasksThread extends Thread {
+    private Exception exception = null;
+    private final Server server;
+    private final RegionServerServices services;
+    private final HRegion region;
+    private final AtomicBoolean signaller;
+
+    PostOpenDeployTasksThread(final HRegion region, final Server server,
+        final RegionServerServices services, final AtomicBoolean signaller) {
+      super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
+      this.setDaemon(true);
+      this.server = server;
+      this.services = services;
+      this.region = region;
+      this.signaller = signaller;
+    }
+
+    public void run() {
+      try {
+        this.services.postOpenDeployTasks(this.region,
+          this.server.getCatalogTracker(), false);
+      } catch (Exception e) {
+        LOG.warn("Exception running postOpenDeployTasks; region=" +
+          this.region.getRegionInfo().getEncodedName(), e);
+        this.exception = e;
+      }
+      // We're done.  Set flag then wake up anyone waiting on thread to complete.
+      this.signaller.set(true);
+      synchronized (this.signaller) {
+        this.signaller.notify();
+      }
+    }
+
+    /**
+     * @return Null or the run exception; call this method after thread is done.
+     */
+    Exception getException() {
+      return this.exception;
+    }
+  }
+
+
+  /**
+   * @param r Region we're working on.
+   * @return Transition znode to OPENED state.
+   * @throws IOException 
+   */
+  private boolean transitionToOpened(final HRegion r) throws IOException {
+    boolean result = false;
+    HRegionInfo hri = r.getRegionInfo();
+    final String name = hri.getRegionNameAsString();
     // Finally, Transition ZK node to OPENED
     try {
-      if (ZKAssign.transitionNodeOpened(server.getZooKeeper(), regionInfo,
-          server.getServerName(), openingVersion) == -1) {
+      if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri,
+          this.server.getServerName(), this.version) == -1) {
         LOG.warn("Completed the OPEN of region " + name +
           " but when transitioning from " +
           " OPENING to OPENED got a version mismatch, someone else clashed " +
           "so now unassigning -- closing region");
-        cleanupFailedOpen(region);
-        return;
+      } else {
+        result = true;
       }
     } catch (KeeperException e) {
       LOG.error("Failed transitioning node " + name +
         " from OPENING to OPENED -- closing region", e);
-      cleanupFailedOpen(region);
-      return;
-    } catch (IOException e) {
-      LOG.error("Failed to close " + name +
-        " after failing to transition -- closing region", e);
-      cleanupFailedOpen(region);
-      return;
     }
+    return result;
+  }
 
-    // Done!  Successful region open
-    LOG.debug("Opened " + name);
+  /**
+   * @return Instance of HRegion if successful open else null.
+   */
+  private HRegion openRegion() {
+    HRegion region = null;
+    try {
+      // Instantiate the region.  This also periodically tickles our zk OPENING
+      // state so master doesn't timeout this region in transition.
+      region = HRegion.openHRegion(this.regionInfo, this.rsServices.getWAL(),
+        this.server.getConfiguration(), this.rsServices,
+        new Progressable() {
+          public void progress() {
+            // We may lose the znode ownership during the open.  Currently its
+            // too hard interrupting ongoing region open.  Just let it complete
+            // and check we still have the znode after region open.
+            tickleOpening("open_region_progress");
+          }
+        });
+    } catch (IOException e) {
+      // We failed open.  Let our znode expire in regions-in-transition and
+      // Master will assign elsewhere.  Presumes nothing to close.
+      LOG.error("Failed open of region=" +
+        this.regionInfo.getRegionNameAsString(), e);
+    }
+    return region;
   }
 
   private void cleanupFailedOpen(final HRegion region) throws IOException {
@@ -200,20 +270,58 @@
public class OpenRegionHandler extends EventHandler {
     this.rsServices.removeFromOnlineRegions(regionInfo.getEncodedName());
   }
 
-  int transitionZookeeperOfflineToOpening(final String encodedName) {
-    // Transition ZK node from OFFLINE to OPENING
+  /**
+   * Transition ZK node from OFFLINE to OPENING.
+   * @param encodedName Name of the znode file (Region encodedName is the znode
+   * name).
+   * @return True if successful transition.
+   */
+  boolean transitionZookeeperOfflineToOpening(final String encodedName) {
     // TODO: should also handle transition from CLOSED?
-    int openingVersion = -1;
     try {
-      if ((openingVersion = ZKAssign.transitionNodeOpening(server.getZooKeeper(),
-          regionInfo, server.getServerName())) == -1) {
-        LOG.warn("Error transitioning node from OFFLINE to OPENING, " +
-            "aborting open");
-      }
+      // Initialize the znode version.
+      this.version =
+        ZKAssign.transitionNodeOpening(server.getZooKeeper(),
+          regionInfo, server.getServerName());
     } catch (KeeperException e) {
-      LOG.error("Error transitioning node from OFFLINE to OPENING for region " +
+      LOG.error("Error transition from OFFLINE to OPENING for region=" +
         encodedName, e);
     }
-    return openingVersion;
+    boolean b = isGoodVersion();
+    if (!b) {
+      LOG.warn("Failed transition from OFFLINE to OPENING for region=" +
+        encodedName);
+    }
+    return b;
+  }
+
+  /**
+   * Update our OPENING state in zookeeper.
+   * Do this so master doesn't timeout this region-in-transition.
+   * @param context Some context to add to logs if failure
+   * @return True if successful transition.
+   */
+  boolean tickleOpening(final String context) {
+    // If previous checks failed... do not try again.
+    if (!isGoodVersion()) return false;
+    String encodedName = this.regionInfo.getEncodedName();
+    try {
+      this.version =
+        ZKAssign.retransitionNodeOpening(server.getZooKeeper(),
+          this.regionInfo, this.server.getServerName(), this.version);
+    } catch (KeeperException e) {
+      server.abort("Exception refreshing OPENING; region=" + encodedName +
+        ", context=" + context, e);
+    }
+    boolean b = isGoodVersion();
+    if (!b) {
+      LOG.warn("Failed refreshing OPENING; region=" + encodedName +
+        ", context=" + context);
+    }
+    return b;
+  }
+
+  private boolean isGoodVersion() {
+    return this.version != -1;
   }
-}
+}
\ No newline at end of file
